{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import re\n",
    "import sys\n",
    "from natsort import natsorted, ns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = dict()\n",
    "\n",
    "with open('/home/felipe/queirozfcom/files/stackoverflow-data-explorer-count-by-tag-2016-2017-taken-on-2017-10-26.txt','r') as f:\n",
    "    for line in f:\n",
    "                \n",
    "        tag,count,yearmonth =line.split()\n",
    "        \n",
    "        tag = tag.strip()\n",
    "        count = count.strip()\n",
    "        yearmonth = yearmonth.strip()\n",
    "\n",
    "        # pad numbers\n",
    "        yearmonth = re.sub(r'^(\\d{4})-(\\d{1})$',r'\\1-0\\2',yearmonth)\n",
    "        \n",
    "        if yearmonth=='2017-10':\n",
    "            continue\n",
    "            # month has not ended \n",
    "        \n",
    "        if results.get(tag,None) is None:\n",
    "            results[tag] = [(yearmonth,int(count) )]\n",
    "        else:\n",
    "            results[tag] = results[tag] + [(yearmonth, int(count) )]            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('2016-01', 3),\n",
       " ('2016-10', 7),\n",
       " ('2016-11', 8),\n",
       " ('2016-12', 6),\n",
       " ('2016-02', 4),\n",
       " ('2016-03', 5),\n",
       " ('2016-04', 2),\n",
       " ('2016-05', 8),\n",
       " ('2016-06', 2),\n",
       " ('2016-07', 2),\n",
       " ('2016-08', 4),\n",
       " ('2016-09', 6),\n",
       " ('2017-01', 6),\n",
       " ('2017-02', 6),\n",
       " ('2017-03', 2),\n",
       " ('2017-04', 9),\n",
       " ('2017-05', 8),\n",
       " ('2017-06', 4),\n",
       " ('2017-07', 7),\n",
       " ('2017-08', 5),\n",
       " ('2017-09', 9)]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results['pymc']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "periods = ['2016-01','2016-02','2016-03','2016-04','2016-05','2016-06',\n",
    "           '2016-07','2016-08','2016-09','2016-10','2016-11','2016-12',\n",
    "           '2017-01','2017-02','2017-03','2017-04','2017-05','2017-06',\n",
    "           '2017-07','2017-08','2017-09']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['pandas',\n",
       " 'spacy',\n",
       " 'tensorflow',\n",
       " 'scrapy',\n",
       " 'theano',\n",
       " 'pymc',\n",
       " 'mxnet',\n",
       " 'pymc3',\n",
       " 'pytorch',\n",
       " 'numpy',\n",
       " 'opencv',\n",
       " 'bokeh',\n",
       " 'gensim',\n",
       " 'beautifulsoup',\n",
       " 'statsmodels',\n",
       " 'keras',\n",
       " 'networkx',\n",
       " 'matplotlib',\n",
       " 'scipy',\n",
       " 'scikit-learn',\n",
       " 'cntk',\n",
       " 'nltk',\n",
       " 'seaborn']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(results.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for key in list(results.keys()):\n",
    "    existing_dates = [pair[0] for pair in results[key]]\n",
    "    \n",
    "    missing = set(periods) - set(existing_dates)\n",
    "    \n",
    "    for period in missing:\n",
    "        results[key] = results[key] + [(period,0)]   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "not enough values to unpack (expected 2, got 1)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-14-bb1bd547dfc8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     17\u001b[0m         \u001b[0mcurrent_counts_for_this_tag\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresults\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtag\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m         \u001b[0mcurrent_pair\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdate\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcnt\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mdate\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcnt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcurrent_counts_for_this_tag\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mdate\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0myearmonth\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     20\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-14-bb1bd547dfc8>\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     17\u001b[0m         \u001b[0mcurrent_counts_for_this_tag\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresults\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtag\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m         \u001b[0mcurrent_pair\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdate\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcnt\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mdate\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcnt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcurrent_counts_for_this_tag\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mdate\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0myearmonth\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     20\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: not enough values to unpack (expected 2, got 1)"
     ]
    }
   ],
   "source": [
    "with open('/home/felipe/queirozfcom/files/crossvalidated-data-explorer-count-by-tag-2016-2017-taken-on-2017-10-26.txt','r') as f:\n",
    "    for line in f:\n",
    "                \n",
    "        tag,count,yearmonth =line.split()\n",
    "\n",
    "        tag = tag.strip()\n",
    "        count = count.strip()\n",
    "        yearmonth = yearmonth.strip()\n",
    "        \n",
    "        # pad numbers\n",
    "        yearmonth = re.sub(r'^(\\d{4})-(\\d{1})$',r'\\1-0\\2',yearmonth)\n",
    "        \n",
    "        if yearmonth=='2017-10':\n",
    "            continue\n",
    "            # month has not ended \n",
    "        \n",
    "        current_counts_for_this_tag = results[tag]\n",
    "        \n",
    "        current_pair = [(date,cnt) for date,cnt in current_counts_for_this_tag if date == yearmonth][0]\n",
    "                          \n",
    "        _,current_count = current_pair\n",
    "                    \n",
    "        new_count = current_count + int(count)\n",
    "        \n",
    "        current_counts_for_this_tag.remove(current_pair)\n",
    "        \n",
    "                \n",
    "        new_pair = (yearmonth,new_count)\n",
    "        \n",
    "        current_counts_for_this_tag.append(new_pair)\n",
    "                                    \n",
    "        results[tag] = current_counts_for_this_tag       \n",
    "        \n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('2016-01', 6),\n",
       " ('2016-10', 5),\n",
       " ('2016-11', 13),\n",
       " ('2016-12', 9),\n",
       " ('2016-02', 6),\n",
       " ('2016-03', 7),\n",
       " ('2016-04', 1),\n",
       " ('2016-05', 8),\n",
       " ('2016-06', 5),\n",
       " ('2016-07', 9),\n",
       " ('2016-08', 7),\n",
       " ('2016-09', 11),\n",
       " ('2017-01', 12),\n",
       " ('2017-02', 17),\n",
       " ('2017-03', 14),\n",
       " ('2017-04', 19),\n",
       " ('2017-05', 20),\n",
       " ('2017-06', 10),\n",
       " ('2017-07', 8),\n",
       " ('2017-08', 20),\n",
       " ('2017-09', 11)]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# merge pymc and pymc3\n",
    "pymc = results['pymc3']\n",
    "pymc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('2016-01', 12),\n",
       " ('2016-10', 10),\n",
       " ('2016-11', 26),\n",
       " ('2016-12', 18),\n",
       " ('2016-02', 12),\n",
       " ('2016-03', 14),\n",
       " ('2016-04', 2),\n",
       " ('2016-05', 16),\n",
       " ('2016-06', 10),\n",
       " ('2016-07', 18),\n",
       " ('2016-08', 14),\n",
       " ('2016-09', 22),\n",
       " ('2017-01', 24),\n",
       " ('2017-02', 34),\n",
       " ('2017-03', 28),\n",
       " ('2017-04', 38),\n",
       " ('2017-05', 40),\n",
       " ('2017-06', 20),\n",
       " ('2017-07', 16),\n",
       " ('2017-08', 40),\n",
       " ('2017-09', 22)]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_pairs = []\n",
    "\n",
    "for (date,count) in results['pymc3']:\n",
    "    \n",
    "    count_pymc = [pair[1] for pair in pymc if pair[0]==date]\n",
    "    \n",
    "    if count_pymc:\n",
    "        new_count = count + count_pymc[0]\n",
    "    else:\n",
    "        new_count = count\n",
    "          \n",
    "    new_pairs.append((date,new_count))\n",
    "    \n",
    "new_pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "results.pop('pymc3',None)\n",
    "results.pop('pymc',None)\n",
    "\n",
    "results['pymc/pymc3'] = new_pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for key in list(results.keys()):\n",
    "    existing_values = [pair[0] for pair in results[key]]\n",
    "    \n",
    "    missing = set(periods) - set(existing_values)\n",
    "    \n",
    "    for period in missing:\n",
    "        results[key] = results[key] + [(period,0)]   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for key in list(results.keys()):\n",
    "    \n",
    "    results[key] = natsorted(results[key],key=lambda pair: pair[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.clf()\n",
    "\n",
    "img = plt.gcf()\n",
    "\n",
    "ax = plt.gca()\n",
    "for key in list(results.keys()):\n",
    "    x = periods\n",
    "    y = [pair[1] for pair in results[key]]\n",
    "    plt.plot(np.array(x),np.array(y),label=key)\n",
    "    \n",
    "plt.legend(loc='lower right')\n",
    "plt.grid(True)\n",
    "\n",
    "plt.yscale('symlog')\n",
    "    \n",
    "img.set_size_inches(20,12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Global TF Kernel (Python 3)",
   "language": "python",
   "name": "global-tf-python-3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
